# -*- coding: utf-8 -*-
# -----------------------------------------
# @Time       : 2020/10/23 8:49
# @Author     : dqz 
# @Email      : duanqizhong@dangdang.com
# @File       : run.py
# @Software   : PyCharm
# -----------------------------------------
import os
# shutil可以操作文件，移动，删除等
import shutil
import time
# urllib form表单提交urlencode:表单数据
import urllib
# 请求模块库，需要安装：pip install requests
import requests
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed

from src.config import USERNAME, PASSWORD, FORM, HOMEPAGE, START_TIME, END_TIME, BASE_DIR, \
    OCTET_STREAM, PARALLEL, MAX_WORKS, QUERY_URL, RE_DOWNLOAD_UNCOMPLETE_FILE

"""
过程描述：
1. 获取登录用户的cookies
2. 依据添加查询所需要的结果
3. 把查询的结果记录到文件内，以便下载完成后作对比，以便发现未下载内容
4. 批量下载，MAX_WORKS是进程数: 已经下载过得，存在同名文件的，跳过下载
5. 对比出来那些文件未成功下载
6. 如果存在未下载完成的文件，移动到undownload目录
7. 重新下载未正确下载的文件
"""

# download 和 undownload文件不存在，先创建目录
if not os.path.isdir(f"{BASE_DIR}/download"):
    os.makedirs(f"{BASE_DIR}/download")
if not os.path.isdir(f"{BASE_DIR}/undownload"):
    os.makedirs(f"{BASE_DIR}/undownload")

def get_current_datetime(serial=False):
    """获取当前时间"""
    if serial:
        date_format = "%Y%m%d%H%M%S"
    else:
        date_format = "%Y-%m-%d %H:%M:%S"
    return time.strftime(date_format, time.localtime())


class Dhub:
    def __init__(self, username=USERNAME, password=PASSWORD):
        self.username = username
        self.password = password
        # 获取cookies
        self.cookies = self.get_cookies()
        # 记录查询结果
        self.query_results = None
        # 记录未下载完成的文件
        self.undownload = None

    def get_cookies(self):
        # 获取cookies
        url = f"{HOMEPAGE}/dhus/login"
        data = {
            "login_username": self.username,
            "login_password": self.password
        }
        try:
            msg = requests.post(url, data=urllib.parse.urlencode(data), headers=FORM)
        except Exception as e:
            raise Exception(f"ERROR:{str(e)}")
        # 获取返回结果内的cookies
        return msg.cookies

    def query(self, start_time=START_TIME, end_time=END_TIME):
        # 这个地方是获取数据的
        # url = f"{HOMEPAGE}/dhus/api/stub/products?filter=(%20beginPosition:[{start_time}%20TO%20{end_time}]%20AND%20endPosition:[{start_time}%20TO%20{end_time}]%20)&offset=0&limit=25&sortedby=ingestiondate&order=desc"
        url = QUERY_URL
        msg = requests.get(url=url, headers=FORM, cookies=self.cookies).json()
        # 做断言，看看是否有记录被查询到，如果未查询到任何结果，报错退出，不在执行后续过程
        assert msg.get("totalresults") > 0, "not query any records!"
        # 记录查询过程
        self.query_results = msg.get("products")

    def download(self, uuid, filename):
        """下载，stream模式"""
        # 输出文件名
        output = f"{BASE_DIR}/download/{filename}.nc"
        url = f"{HOMEPAGE}/dhus/odata/v1/Products('{uuid}')/$value"

        print(f"async download >>> {url} {filename}")
        # 数据流方式获取数据
        mystream = requests.get(url, headers=OCTET_STREAM, cookies=self.cookies, stream=True)
        with open(output, "wb") as fw:
            print("wait download .", end="")
            tj = 1
            # 每次下载chunk_size数据大小后写入文件内
            for chunk in mystream.iter_content(chunk_size=4096):
                if tj % 100 == 0:
                    print()
                if chunk:
                    fw.write(chunk)
                # print主要是看是否在执行下载，如果在不断下载，每次达到chunk_size大小会输出一个.
                print('.', end='')
                tj += 1
        print(f"already download <<< {url}")
        # 已下载的文件名，追加到records_download.txt内
        with open(f"{BASE_DIR}/records_download.txt", "a") as fw:
            fw.write(f"{filename}\n")

    def download_pool(self, parallel=True):
        # 进程池下载，如果单个进程，设置parallel=False
        with ProcessPoolExecutor(max_workers=MAX_WORKS) as executor:
            # 从已下载的文件内查看已经下载了那些文件，如果文件不存在，出事为空
            already_downloads = []
            if os.path.exists(f"{BASE_DIR}/records_download.txt"):
                # 如果存在下载的记录已下载的
                already_downloads = [filename.strip() for filename in open(f"{BASE_DIR}/records_download.txt", "r") if
                                     filename]
            futures=[]
            for info in self.query_results:
                uuid = info.get("uuid")
                filename = info.get("identifier")
                # 判断当前要下载的文件，是否在已下载列表内，如果是continue继续下一个
                if filename in already_downloads:
                    print(f"records_download.txt already download it: {filename}")
                    continue
                if parallel:
                    # 添加到进程池内
                    task=executor.submit(self.download, uuid, filename)
                    futures.append(task)
                else:
                    # 调用方法的方式下载
                    self.download(uuid, filename)
                    # break
            if parallel and futures:
                # 获取result，阻塞进程，直到执行完
                for task in as_completed(futures):
                    try:
                        task.result()
                    except Exception as e:
                        print(e)


    def write_query_info_to_file(self):
        """查询条件写入文件内，便于核查"""
        with open(f"{BASE_DIR}/records.txt_{get_current_datetime(True)}", 'w') as fw:
            # 记录这次查询的url
            fw.write(QUERY_URL)
            # 记录格式：UUID identifier:和文件名一样，就是没有后缀 date
            fw.write("\nUUID\tidentifier\tsummary[Date]\n")
            for info in self.query_results:
                fw.write(f"{info.get('uuid')}\t{info.get('identifier')}\t{info.get('summary')[0]}\n")

    def compare(self):
        # 对比那些文件已下载，那些未下载
        path_records_download = os.path.join(BASE_DIR, "records_download.txt")
        # 判断records_download.txt已下载内容文件存不存在
        if not os.path.exists(path_records_download):
            raise Exception(f"not found file:{path_records_download}, please check filename!")

        # 最后一次下载的文件：日期一定比其他日期都大：这个文件不用判断存不存在，一定存在的
        last_query_records = self.get_last_query_records_file()
        # 如果想对比前面的文件，可以注释上一行，吧名字写死，这里需要全路径
        # last_query_records = os.path.join(BASE_DIR,"要对比的文件名字")
        print(f"最后执行的文件是：{last_query_records}")

        # 读取已下载的文件
        download_files = [file.strip() for file in open(path_records_download) if file]
        # 读取查询条件查出的文件
        query_results = []
        i = 0
        for line in open(last_query_records):
            # 去掉前两行，一行记录的查询条件，一行记录的格式
            if i < 2:
                i += 1
                continue
            # 空格分隔，如果有问题，换为 line.split("\t")写入的时候是按\t分隔的
            uuid, idenfier, date = line.split("\t")
            if idenfier:
                # idenfier不为空记录
                query_results.append(idenfier)
        # 转化为集合做集合运算
        diff_result = set(query_results) - set(download_files)

        print(f"未成功下载的文件:")
        for diff in diff_result:
            print(diff)
        self.undownload = diff_result

    def get_last_query_records_file(self):
        # 获取BASE_DIR：dhub下文件名是records.txt_开头的文件名
        files = [file for file in os.listdir(BASE_DIR) if file.startswith("records.txt_")]
        max_file = files[0]
        for file in files[1:]:
            # 文件名进行字符串比较，时间大的比较大
            if file > max_file:
                max_file = file
        return os.path.join(BASE_DIR, max_file.strip())

    def rename_undownload(self):
        download_dir = os.path.join(BASE_DIR, "download")
        undownload_dir = os.path.join(BASE_DIR, "undownload")
        for filename in self.undownload:
            undownload_complate_file = f"{download_dir}/{filename}.nc"
            undownload_rename_file = f"{undownload_dir}/{filename}.nc"
            if os.path.exists(undownload_complate_file):
                # 把未下载完成的文件从download目录移动到undownload目录下
                print(f"移动：{filename}.nc 到 undownload目录下")
                shutil.move(undownload_complate_file, undownload_rename_file)

    def redownload(self):
        for filename in self.undownload:
            for info in self.query_results:
                info_filename = info.get("identifier")
                if filename == info_filename:
                    uuid = info.get("uuid")
                    self.download(uuid, filename)

    def run(self, parallel=PARALLEL):
        # 查询记录
        self.query()
        # 查询结果记录文件
        self.write_query_info_to_file()
        # 开始下载，下载的记录文件:
        self.download_pool(parallel=parallel)
        # 对比这次执行的，那些文件未下载
        self.compare()
        # 对于未下载完成的文件，进行重命名，添加后缀.undownload
        self.rename_undownload()
        # 重下载为成功下载的文件：配置文件配置False为不进行重新下载
        if RE_DOWNLOAD_UNCOMPLETE_FILE:
            self.redownload()
            # 再次对比
            self.compare()
            # 再次把未成功下载成功的文件移动到undownload目录下
            self.rename_undownload()


if __name__ == "__main__":
    dhub = Dhub()
    dhub.run()
